<?php
class eCrawler {
  
  var $website = '';
  var $headers = array();
  var $proxy_ip = array();
  var $useragents = array();
  var $use_proxy = 0;
  var $error_debug = 0;
  var $crawl_all_links = 0;
  var $debug_curl = 0;
  var $dataset_of_links = array();
  var $all_comb_links = array();

  private $htmldata = '';
  private $curldata = array();
  private $dont_crawl	= array('pdf', 'jpg', 'png', 'jpeg','zip', 'gz', 'tar', 'txt','gif','js','css');
  private $ignore_links = array('javascript:','PHPSESSID=','#','',';jsessionid=','.swf','.pdf','mailto:','.jpg','.jpeg','.gif','.png','@','.zip','.exe','.txt');

  //contructor is called
  function eCrawler($params = array()) {
    set_time_limit(0);
    $this->proxy_ip = array('108.70.16.241:80', '109.105.84.70:8008', '109.121.196.2:3128',
                            '109.121.235.186:8080', '109.121.236.6:8080', '109.123.70.47:80',
                            '109.237.147.41:80', '109.235.49.224:3128', '109.86.220.235:3128',
                            '110.136.149.35:8080', '110.136.178.208:80', '110.136.207.55:8000',
                            '110.137.124.40:3128', '110.137.127.142:8080');
    
    $this->useragents = array('Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.8) Gecko/2009032609 Firefox/3.0.8',
                              'Mozilla/4.8 (Windows; U; Windows NT 4.1; en-US; rv:1.9.0.8) Gecko/2009032609 Firefox/3.0.6',
                              'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)',
                              'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0)',
                              'Mozilla/3.0 (Win95; I)',
                              'Mozilla/3.01 (Macintosh; PPC)',
                              'Mozilla/4.0 (compatible; MSIE 4.01; AOL 4.0; Windows 98)',
                              'Mozilla/5.0 (compatible; Konqueror/2.2.2; Linux 2.4.14-xfs; X11; i686)',
                              'Mozilla/5.0 (Windows; U; Win98; en-US; rv:0.9.2) Gecko/20010726 Netscape6/6.1',
                              'Opera/6.x (Windows NT 4.0; U) [de]',
                              'Opera/7.x (Windows NT 5.1; U) [en]',
                               );
    $this->set_global_values($params);
  }

  //Set global/class values to the class
  function set_global_values($params = array()) {    
    foreach ($params as $key => $val) {      
      if (isset($this->$key)) {
        $this->$key = $val;
      }
    }
  }

  //Start the script to crawl
  public function run_script() {
    $this->get_page();    
    $dataset = array();
    $dataset_recv = array();
    
    $dataset['anchor_links']      = $this->get_anchor_links();
    $dataset['page_titles']       = $this->get_page_titles();
    $dataset['metadata']          = $this->get_metadata();
    $dataset['meta_description']  = $this->get_meta_description();
    $dataset['meta_keywords']     = $this->get_meta_keywords();
    $dataset['meta_google_verify']= $this->get_meta_google_verify();
    $dataset['meta_yahoo_verify'] = $this->get_meta_yahoo_verify();
    $dataset['meta_robots']       = $this->get_meta_robots();
    $dataset['h1_data']           = $this->get_h1_data();
    $dataset['h2_data']           = $this->get_h2_data();
    $dataset['h3_data']           = $this->get_h3_data();
    $dataset['strong_data']       = $this->get_strong_data();
    $dataset['b_data']            = $this->get_b_data();
    $dataset['body_data']         = $this->get_body_data();
	$dataset['img_data']          = $this->get_img_data();
	$dataset['links_data']        = $this->get_links_data();
	$dataset['links_data_checks'] = $this->get_links_data_checks();	
	
    $dataset['curl_data']         = $this->curldata;
	
    $this->buff_op('Site crawling: '.$this->website);
    $this->buff_op($dataset);
    $this->buff_op('<hr/>');
    $this->buff_op('Status crawl_all_links: '.$this->crawl_all_links);
    if($this->crawl_all_links == 1){
      $filter_links = array();
      $anchor_links = $dataset['anchor_links'];
      $anchor_links = array_unique($anchor_links);
      
      if (is_array($anchor_links) && count($anchor_links) > 1) {        
        $filter_links = $this->filter_link($anchor_links, $this->website);        
        //now run deep link crawling for all the filter links
        if(count($filter_links) > 0){
          $this->buff_op('Run filter_links');
          foreach ($filter_links as $iter => $links) {
            $this->buff_op('Crawl site: '.$links);
            $this->website = $links;
            $this->get_page();
            $dataset_recv[$links]['anchor_links']      = $this->get_anchor_links();
            $dataset_recv[$links]['page_titles']       = $this->get_page_titles();
            $dataset_recv[$links]['metadata']          = $this->get_metadata();
            $dataset_recv[$links]['meta_description']  = $this->get_meta_description();
            $dataset_recv[$links]['meta_keywords']     = $this->get_meta_keywords();
            $dataset_recv[$links]['meta_google_verify']= $this->get_meta_google_verify();
            $dataset_recv[$links]['meta_yahoo_verify'] = $this->get_meta_yahoo_verify();
            $dataset_recv[$links]['meta_robots']       = $this->get_meta_robots();
            $dataset_recv[$links]['h1_data']           = $this->get_h1_data();
            $dataset_recv[$links]['h2_data']           = $this->get_h2_data();
            $dataset_recv[$links]['h3_data']           = $this->get_h3_data();
            $dataset_recv[$links]['strong_data']       = $this->get_strong_data();
            $dataset_recv[$links]['b_data']            = $this->get_b_data();
            $dataset_recv[$links]['body_data']         = $this->get_body_data();
            $this->buff_op('<hr/>');
            //only run for 3 iterations for testing
            if($iter >= 3){
              break;
            }
          }
        }
      }
    }
	return $dataset; 
  }
  
  private function set_dataset_of_links($anchor_links = ''){
    if (is_array($anchor_links) && count($anchor_links) > 1) {
      foreach ($anchor_links as $iter => $links) {

        $found_ignore_links = false;
        foreach ($this->ignore_links as $pattern) {
          if (stripos($links, $pattern) !== FALSE) {
            $found_ignore_links = true;
            break;
          }
        }
        //if there is a link that is from the ignore list then skip it
        if($found_ignore_links == true){
          continue;
        }

        //also manually ignore the javascript and # links
        if(preg_match('/.*?javascript:void\(0\)/', $links) || preg_match('/^#/', $links)){
          continue;
        }

        //check the extension with the ignore list
        $extension = strtolower(end(explode(".", $links)));
        if(!in_array($end, $this->dont_crawl))
        {
          $filter_links[] = $links;
          //echo '<br/>filter => '.$links;
        }

        $this->dataset_of_links[] = $links;
      }
    }
  }

  private function filter_link($anchor_links = array(), $url = ''){
    $filter_links = array();
    if (is_array($anchor_links) && count($anchor_links) > 1) {
      foreach ($anchor_links as $iter => $links) {

        $found_ignore_links = false;
        foreach ($this->ignore_links as $pattern) {
          if (stripos($links, $pattern) !== FALSE) {
            $found_ignore_links = true;
            break;
          }
        }
        //if there is a link that is from the ignore list then skip it
        if($found_ignore_links == true){
          continue;
        }

        //also manually ignore the javascript and # links
        if(preg_match('/.*?javascript:void\(0\)/', $links) || preg_match('/^#/', $links)){
          continue;
        }

        //check if Http protocol is set or not
        if(!preg_match('/http(s)?(.*)/', $links)){
          //set the links with a base url
          //if there is not slash then add the base url with a slash
          if(preg_match('/^\//', $links)){
            $links = $url . $links;
          }else{
            $links = $url . '/' . $links;
          }
        }

        //check the base link is same as the new link
        if($url == $links || $url . '/' == $links){
          continue;
        }

        //check the extension with the ignore list
        $extension = strtolower(end(explode(".", $links)));
        if(!in_array($end, $this->dont_crawl))
        {
          $filter_links[] = $links;
          //echo '<br/>filter => '.$links;
        }
      }
    }
    return $filter_links;
  }

  private function buff_op($text = ''){
  	if($this->debug_curl == 1)
	{
		ob_start();
		if(is_array($text)){
		  echo '<pre>';
		  print_r($text);
		  echo '</pre>';
		}else{
		  echo '<br/>'.$text;
		}
		ob_flush();
		flush();
	}
  }

  private function get_anchor_links(){
    preg_match_all("/<a[ \r\n\t]{1}[^>]*HREF[^=]*=[ '\"\n\r\t]*([^ \"'>\r\n\t#]+)[ \"'>\r\n\t#>][^>]*>(.*)<\/a[ \r\n\t]*>/isU",$this->htmldata,$anchor_links);
    $anchor_links = (!empty($anchor_links[1])) ? $anchor_links[1]:'';//all links will be in element 1
    return $anchor_links;
  }

  private function get_page_titles(){
    preg_match_all('/<title>(.*?)\<\/title\>/si', $this->htmldata, $page_titles);
    $page_titles = (!empty($page_titles[1]))?$page_titles[1]:'';//all data will be in element 1
    return $page_titles;
  }

  private function get_metadata(){
    preg_match_all('/<meta (.*?) \/>/si', $this->htmldata, $metadata);
    $metadata = (!empty($metadata[1])) ? $metadata[1]:'';//all data will be in element 1
    return $metadata;
  }

  private function get_meta_description(){
    preg_match_all('/<meta .+?description\" content=\"(.*?)\"/si', $this->htmldata, $meta_description);
    $meta_description = (!empty($meta_description[1])) ? $meta_description[1]:'';//all data will be in element 1
    return $meta_description;
  }

  private function get_meta_keywords(){
    preg_match_all('/<meta .+?keywords" content=\"(.*?)\"/si', $this->htmldata, $meta_keywords);
    $meta_keywords = (!empty($meta_keywords[1])) ? $meta_keywords[1]:'';//all data will be in element 1
    return $meta_keywords;
  }

  private function get_meta_google_verify(){
    preg_match_all('/<meta .+?verify-v1" content=\"(.*?)\"/si', $this->htmldata, $meta_google_verify);
    $meta_google_verify = (!empty($meta_google_verify[1])) ? $meta_google_verify[1]:'';//all data will be in element 1
    return $meta_google_verify;
  }

  private function get_meta_yahoo_verify(){  	
    preg_match_all('/<meta .+?y_key" content=\"(.*?)\"/si', $this->htmldata, $meta_yahoo_verify);
    $meta_yahoo_verify = (!empty($meta_yahoo_verify[1])) ? $meta_yahoo_verify[1]:'';//all data will be in element 1
    return $meta_yahoo_verify;
  }

  private function get_meta_robots(){
    preg_match_all('/<meta .+?robots" content=\"(.*?)\"/si', $this->htmldata, $meta_robots);
    $meta_robots = (!empty($meta_robots[1])) ? $meta_robots[1]:'';//all data will be in element 1
    return $meta_robots;
  }

  private function get_body_data(){
    preg_match_all('/<body(.*?)>(.*?)\<\/body>/si', $this->htmldata, $body_data);
    $body_data = (!empty($body_data[2])) ? $body_data[2]:'';//all data will be in element 2
    return $body_data;
  }

  private function get_h1_data(){
    preg_match_all('/<h1(.*?)>(.*?)\<\/h1>/si', $this->htmldata, $h1_data);
    $h1_data = $h1_data[2];//all data will be in element 2
    $h1_data = array_map("strip_tags", $h1_data);
    return $h1_data;
  }

  private function get_h2_data(){
    preg_match_all('/<h2(.*?)>(.*?)\<\/h2>/si', $this->htmldata, $h2_data);
    $h2_data = $h2_data[2];//all data will be in element 2
    $h2_data = array_map("strip_tags", $h2_data);
    return $h2_data;
  }

  private function get_h3_data(){
    preg_match_all('/<h3(.*?)>(.*?)\<\/h3>/si', $this->htmldata, $h3_data);
    $h3_data = $h3_data[2];//all data will be in element 2
    $h3_data = array_map("strip_tags", $h3_data);
    return $h3_data;
  }

  private function get_strong_data(){
    preg_match_all('/<strong(.*?)>(.*?)\<\/strong>/si', $this->htmldata, $strong_data);
    $strong_data = $strong_data[2];//all data will be in element 2
    $strong_data = array_map("strip_tags", $strong_data);
    return $strong_data;
  }

  private function get_b_data(){
    preg_match_all('/<b(.*?)>(.*?)\<\/b>/si', $this->htmldata, $b_data);
    $b_data = $b_data[2];//all data will be in element 2
    $b_data = array_map("strip_tags", $b_data);
    return $b_data;
  }
  
  private function get_img_data(){
  	$img_data = array();
	$incomp_recom = array();
    preg_match_all('/<img(.*?)(.*?)>/si', $this->htmldata, $img_data);	
    $img_data = $img_data[0];//all data will be in element 0
	$img_data = array_unique($img_data);
	$recom = array();
	if(is_array($img_data) && count($img_data) > 0){
		foreach($img_data as $key => $img){			
			preg_match_all('/title=[\'\"](.*?)[\'\"](.*?)/si', $img,$recom_title);
			preg_match_all('/alt=[\'\"](.*?)[\'\"](.*?)/si', $img,$recom_alt);
			preg_match_all('/width=[\'\"](.*?)[\'\"](.*?)/si', $img,$recom_width);
			preg_match_all('/height=[\'\"](.*?)[\'\"](.*?)/si', $img,$recom_height);
			preg_match_all('/src=[\'\"](.*?)[\'\"](.*?)/si', $img,$recom_src);
			
			$recom[$key]['img'] = $img;//img
			$recom[$key]['title'] = (isset($recom_title[1][0]))?$recom_title[1][0]:'--';//all data will be in element 1
			$recom[$key]['alt'] = (isset($recom_alt[1][0]))?$recom_alt[1][0]:'--';//all data will be in element 1
			$recom[$key]['width'] = (isset($recom_width[1][0]))?$recom_width[1][0]:'--';//all data will be in element 1
			$recom[$key]['height'] = (isset($recom_height[1][0]))?$recom_height[1][0]:'--';//all data will be in element 1
			$recom[$key]['src'] = (isset($recom_src[1][0]))?$recom_src[1][0]:'--';//all data will be in element 1
			
			$flag_incomplete = 0;
			if(empty($recom[$key]['title']))
				$flag_incomplete = 1;
			if(empty($recom[$key]['alt']))
				$flag_incomplete = 1;
			if(empty($recom[$key]['src']))
				$flag_incomplete = 1;
			
			if($flag_incomplete == 1){
				$incomp_recom[] = htmlentities($img);
			}				
		}
	}
	
	$incomp_recom = array_unique($incomp_recom);
	
	$img_d = array();
    $img_d['img'] = $img_data;
	$img_d['recom'] = $recom;
	$img_d['cnt_recom'] = count($recom);
	$img_d['incomp_recom'] = $incomp_recom;
	$img_d['cnt_incomp_recom'] = count($incomp_recom);
    return $img_d;
  }
  
  private function get_links_data(){
    
    $anchor_links = array();
	preg_match_all("/<a[ \r\n\t]{1}[^>]*HREF[^=]*=[ '\"\n\r\t]*([^ \"'>\r\n\t#]+)[ \"'>\r\n\t#>][^>]*>(.*)<\/a[ \r\n\t]*>/isU",$this->htmldata,$anchor_links);	
    $anchor_links = $anchor_links[0];//all data will be in element 1
	$recom = array();
	$cnt_title_missing = 0;
	if(is_array($anchor_links) && count($anchor_links) > 0){
		foreach($anchor_links as $key => $img){
			preg_match_all('/title=[\'"\n\r\t](.*?)[\'\"\n\r\t](.*?)/si', $img,$recom_title);
			preg_match_all('/href=[\'"\n\r\t](.*?)[\'\"\n\r\t](.*?)/si', $img,$recom_src);			
			
			//first check whether the data is empty or not
			$title = (isset($recom_title[1][0]) && !empty($recom_title[1][0])) ? $recom_title[1][0]:'';
			$aurl = (isset($recom_src[1][0]) && !empty($recom_src[1][0])) ? $recom_src[1][0]:'';
			
			//now check if title is empty then its nad for SEO
			if(empty($title)){
				if(!empty($aurl)){
					$recom[$img] = $aurl;
					$cnt_title_missing++;
				}
			}
		}
	}
	$link_d = array();
    $link_d['links'] = $anchor_links;
	$link_d['recom'] = $recom;	
	$link_d['recom_cnt'] = $cnt_title_missing;
    return $link_d;
  }
  
  private function get_links_data_checks(){
    
    $anchor_links = array();
	//this will ive us the path or the host of the website we are crawling
	$path_parts = parse_url($this->website);
	if(isset($path_parts['host']) && !empty($path_parts['host']))
		$path_parts['host'] = $path_parts['host'];
	if(isset($path_parts['path']) && !empty($path_parts['path']))
		$path_parts['host'] = $path_parts['path'];	
	else	
		$path_parts['host'] = $this->website;
		
	preg_match_all("/<a[ \r\n\t]{1}[^>]*HREF[^=]*=[ '\"\n\r\t]*([^ \"'>\r\n\t#]+)[ \"'>\r\n\t#>][^>]*>(.*)<\/a[ \r\n\t]*>/isU",$this->htmldata,$anchor_links);		
    $anchor_links = $anchor_links[1];//all data will be in element 1
	$anchor_links = array_unique($anchor_links);
	$recom = array();
	$recom_int = array();
	$recom_ext = array();
	
	$cnt_int_link = 0;//Count of internal links
	$cnt_ext_link = 0;//Count of external links
	if(is_array($anchor_links) && count($anchor_links) > 0){
		foreach($anchor_links as $key => $img){
		
			//skip this part and continue with the loop
			$chk_hash_url = preg_match('/#/si', $img);
			if($chk_hash_url == true ) continue;
			$chk_js_url = preg_match('/javascript(:)?/si', $img);
			if($chk_js_url == true ) continue;			
			
			//to check the internal link, first we check whether its HTTP or HTTPS link
			//Then we check with the host url if its from the same domain
			$chk_int_link = 0;
			$chk_local_url = preg_match('/http(s)?:\/\//si', $img);
			
			$img_parts = parse_url($img);
			
			if(strpos($img,$path_parts['host'])){
				$chk_int_link = 1;
			}
			else if(!isset($img_parts['host'])){
				$chk_int_link = 1;
			}
			
			if($chk_int_link == 1){
				$recom_int[] = $img;//true if its an internal link
				$cnt_int_link++;
			}else{				
				$recom_ext[] = $img;//false for external link
				$cnt_ext_link++;
			}	
		}
	}
	
	$link_d = array();
    $link_d['links'] = $anchor_links;
	$link_d['recom_int'] = $recom_int;	
	$link_d['recom_ext'] = $recom_ext;		
	$link_d['cnt_int_link'] = $cnt_int_link;
	$link_d['cnt_ext_link'] = $cnt_ext_link;
    return $link_d;
  }   


  //check if the functionas called exists or not
  public function __call($func,$val) {
    if(!isset($this->$func)){
      $this->show_error('Sorry, function "'.$func.'" was not defined in the class.');
    }
  }

  //get the page contents using curl
  public function get_page() {
    $url = $this->website;
    $proxy_ip = $this->proxy_ip;
    $useragents = $this->useragents;
    $use_proxy = $this->use_proxy;

    if(in_array($url, $this->all_comb_links)) break;

    $this->all_comb_links[] = $url;

    if (function_exists('curl_init')) {      
      if(!preg_match('/^http(s)?:\/\//', $url)){
        $url = 'http://'.$url;
      }      
      $proxy_ip = $proxy_ip[array_rand($proxy_ip)];
      $useragents = $useragents[array_rand($useragents)];
      /*
        curl_setopt($ch, CURLOPT_HTTPPROXYTUNNEL, 1);
        curl_setopt($ch, CURLOPT_PROXY, '128.114.63.15:3128');
       */
      $ch = curl_init($url);
      
      $options = array(
        CURLOPT_REFERER        => $url,     // web page to crawl
        CURLOPT_RETURNTRANSFER => true,     // return web page
        CURLOPT_HEADER         => false,    // don't return headers
        CURLOPT_FOLLOWLOCATION => true,     // follow redirects
        CURLOPT_ENCODING       => "",       // handle compressed
        CURLOPT_USERAGENT      => $useragents, // who am i
        CURLOPT_AUTOREFERER    => false,     // set referer on redirect
        CURLOPT_CONNECTTIMEOUT => 120,      // timeout on connect
        CURLOPT_TIMEOUT        => 120,      // timeout on response
        CURLOPT_MAXREDIRS      => 10,       // stop after 10 redirects
        CURLOPT_FOLLOWLOCATION => true,       // stop after 10 redirects
      );

      if($use_proxy == true){
        $options[CURLOPT_HTTPPROXYTUNNEL] = $use_proxy;
        $options[CURLOPT_PROXY] = $proxy_ip;
      }

      $ch      = curl_init( $url );
      curl_setopt_array( $ch, $options );
      
      $output = curl_exec( $ch );
      $info   = curl_getinfo( $ch );
      $errno  = curl_errno($ch);
      $error  = curl_error($ch);
      
      $this->curldata[$url]['info']   = $info;
      $this->curldata[$url]['errno']  = $errno;
      $this->curldata[$url]['error']  = $error;
      $this->curldata[$url]['output']  = $output;
	 
	  $this->curldata['last_op']['info']   = $info;
      $this->curldata['last_op']['errno']  = $errno;
      $this->curldata['last_op']['error']  = $error;
      $this->curldata['last_op']['output']  = $output;		

      if($errno == 0){
        $this->htmldata = $output;
      }else{
        $str = '';
        $str .= '<br/>Sorry page does not exists -> '.$url;
        $str .= '<br/>errno -> '.$errno;
        $str .= '<br/><br/>error -> '.$error;
        $str .= '<br/><br/>info -> ';
        foreach($info as $k => $v){
          $str .= '<br/>&nbsp;&nbsp;'.$k.' -> '.$v;
        }
        $this->show_error($str);        
      }
    } else {
      return @file_get_contents($url);
    }
  }

  //show internal error
  public function show_error($err = ''){
    $str = '<div style="background: none repeat scroll 0 0 #FFFFFF;width:800px;position: fixed; top: 0px; left: 15%; margin:25px 0 0 0;font:12px Arial,Helvetica,sans-serif; font-weight: normal;padding: 20px 15px 20px 15px;border:1px solid #CC0000; color:#CC0000;">
              <h6 style="font-size: 14px; font-weight: bold;padding:0;margin:0;">Errors:</h6>
              <p style="padding-top: 10px;">'.$err.'</p>
            </div>';
	if($this->error_debug == 1)		
	    echo $str;
  }
}//</EOC-eCrawler>
?>